<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>prometheus+grafana+alertmanager监控k8s无坑版 | Alexcld</title><meta name="keywords" content="prometheus+grafana+alertmanager监控k8s无坑版"><meta name="author" content="Alex"><meta name="copyright" content="Alex"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta http-equiv="Cache-Control" content="no-transform"><meta http-equiv="Cache-Control" content="no-siteapp"><meta name="description" content="prometheus+grafana+alertmanager监控k8s无坑版摘要k8s搭建完成并正常使用的基础上，需要有一个动态存储我的环境：k8s版本    | Kubeadm部署 v1.18.0——– | —–k8s-master  | 172.22.254.57k8s-node1  | 172.22.254.62k8s-node2  | 172.22.254.63(nfs服务端)Stora">
<meta property="og:type" content="article">
<meta property="og:title" content="prometheus+grafana+alertmanager监控k8s无坑版">
<meta property="og:url" content="http://alexcld.com/posts/34136.html">
<meta property="og:site_name" content="Alexcld">
<meta property="og:description" content="prometheus+grafana+alertmanager监控k8s无坑版摘要k8s搭建完成并正常使用的基础上，需要有一个动态存储我的环境：k8s版本    | Kubeadm部署 v1.18.0——– | —–k8s-master  | 172.22.254.57k8s-node1  | 172.22.254.62k8s-node2  | 172.22.254.63(nfs服务端)Stora">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img.alexcld.com/img/214606-15553359668cc5.jpg">
<meta property="article:published_time" content="2021-07-21T04:37:08.000Z">
<meta property="article:modified_time" content="2021-08-04T06:41:01.853Z">
<meta property="article:author" content="Alex">
<meta property="article:tag" content="prometheus+grafana+alertmanager监控k8s无坑版">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://img.alexcld.com/img/214606-15553359668cc5.jpg"><link rel="shortcut icon" href="https://img.alexcld.com/img/author.png"><link rel="canonical" href="http://alexcld.com/posts/34136"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//fonts.googleapis.com" crossorigin="crossorigin"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?ed29f24d552b68e914260fcff2860767";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Titillium+Web&amp;display=swap"><script>var GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  ClickShowText: {"text":"I,LOVE,YOU","fontSize":"15px"},
  lightbox: 'fancybox',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#121212","position":"bottom-left"},
  justifiedGallery: {
    js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
    css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isanchor: false
};

var saveToLocal = {
  set: function setWithExpiry(key, value, ttl) {
    const now = new Date()
    const expiryDay = ttl * 86400000
    const item = {
      value: value,
      expiry: now.getTime() + expiryDay,
    }
    localStorage.setItem(key, JSON.stringify(item))
  },

  get: function getWithExpiry(key) {
    const itemStr = localStorage.getItem(key)

    if (!itemStr) {
      return undefined
    }
    const item = JSON.parse(itemStr)
    const now = new Date()

    if (now.getTime() > item.expiry) {
      localStorage.removeItem(key)
      return undefined
    }
    return item.value
  }
}</script><script id="config_change">var GLOBAL_CONFIG_SITE = { 
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2021-08-04 14:41:01'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><link rel="stylesheet" href="APlayer.min.css"><div id="aplayer"></div><script src="https://cdn.jsdelivr.net/npm/meting@2.0.1/dist/Meting.min.js" async></script><script>(function () {  window.activateDarkMode = function () {
    document.documentElement.setAttribute('data-theme', 'dark')
    if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
    }
  }
  window.activateLightMode = function () {
    document.documentElement.setAttribute('data-theme', 'light')
   if (document.querySelector('meta[name="theme-color"]') !== null) {
      document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
    }
  }
  const autoChangeMode = '2'
  const t = saveToLocal.get('theme')
  if (autoChangeMode === '1') {
    const isDarkMode = window.matchMedia('(prefers-color-scheme: dark)').matches
    const isLightMode = window.matchMedia('(prefers-color-scheme: light)').matches
    const isNotSpecified = window.matchMedia('(prefers-color-scheme: no-preference)').matches
    const hasNoSupport = !isDarkMode && !isLightMode && !isNotSpecified
    if (t === undefined) {
      if (isLightMode) activateLightMode()
      else if (isDarkMode) activateDarkMode()
      else if (isNotSpecified || hasNoSupport) {
        const now = new Date()
        const hour = now.getHours()
        const isNight = hour <= 6 || hour >= 18
        isNight ? activateDarkMode() : activateLightMode()
      }
      window.matchMedia('(prefers-color-scheme: dark)').addListener(function (e) {
        if (saveToLocal.get('theme') === undefined) {
          e.matches ? activateDarkMode() : activateLightMode()
        }
      })
    } else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else if (autoChangeMode === '2') {
    const now = new Date()
    const hour = now.getHours()
    const isNight = hour <= 6 || hour >= 18
    if (t === undefined) isNight ? activateDarkMode() : activateLightMode()
    else if (t === 'light') activateLightMode()
    else activateDarkMode()
  } else {
    if (t === 'dark') activateDarkMode()
    else if (t === 'light') activateLightMode()
  }const asideStatus = saveToLocal.get('aside-status')
if (asideStatus !== undefined) {
   if (asideStatus === 'hide') {
     document.documentElement.classList.add('hide-aside')
   } else {
     document.documentElement.classList.remove('hide-aside')
   }
}})()</script><link rel="stylesheet" href="/css/flipcountdown.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/sviptzk/StaticFile_HEXO@latest/butterfly/css/pool.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/sviptzk/StaticFile_HEXO@latest/butterfly/css/iconfont.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/alexclownfish/hexo-gitcalendar@v1.0/gitcalendar.css"/><meta name="generator" content="Hexo 5.2.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="author-avatar"><img class="avatar-img" src="https://img.alexcld.com/img/author.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">24</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">24</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/download"><i class="fa-fw fa fa-download"></i><span> 下载</span></a></div><div class="menus_item"><a class="site-page" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 追剧怪</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 娱乐</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page" href="/books/"><i class="fa-fw fa fa-heartbeat"></i><span> 书架</span></a></li><li><a class="site-page" href="/Gallery/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li><li><a class="site-page" href="/tgxxl/"><i class="fa-fw fa fa-gamepad"></i><span> 糖果消消乐</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 博主</span></a></div></div></div></div><div id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url(https://img.alexcld.com/img/214606-15553359668cc5.jpg)"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">Alexcld</a></span><span id="weather-v2-plugin-simple"></span><span id="menus"><div id="search_button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/download"><i class="fa-fw fa fa-download"></i><span> 下载</span></a></div><div class="menus_item"><a class="site-page" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 追剧怪</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 娱乐</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page" href="/books/"><i class="fa-fw fa fa-heartbeat"></i><span> 书架</span></a></li><li><a class="site-page" href="/Gallery/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li><li><a class="site-page" href="/tgxxl/"><i class="fa-fw fa fa-gamepad"></i><span> 糖果消消乐</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 博主</span></a></div></div><span class="close" id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></span></span></nav><div id="post-info"><h1 class="post-title">prometheus+grafana+alertmanager监控k8s无坑版</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2021-07-21T04:37:08.000Z" title="发表于 2021-07-21 12:37:08">2021-07-21</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2021-08-04T06:41:01.853Z" title="更新于 2021-08-04 14:41:01">2021-08-04</time></span></div><div class="meta-secondline"> <span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">4.4k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>23分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="prometheus-grafana-alertmanager监控k8s无坑版"><a href="#prometheus-grafana-alertmanager监控k8s无坑版" class="headerlink" title="prometheus+grafana+alertmanager监控k8s无坑版"></a>prometheus+grafana+alertmanager监控k8s无坑版</h1><h2 id="摘要"><a href="#摘要" class="headerlink" title="摘要"></a>摘要</h2><p>k8s搭建完成并正常使用的基础上，需要有一个动态存储<br>我的环境：<br>k8s版本    | Kubeadm部署 v1.18.0<br>——– | —–<br>k8s-master  | 172.22.254.57<br>k8s-node1  | 172.22.254.62<br>k8s-node2  | 172.22.254.63(nfs服务端)<br>StorageClass | nfs-storage</p>
<p>k8s-master有污点，如果需要监控到master，去除污点即可（非必要）</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl taint nodes node1 key1&#x3D;value1:NoSchedule-</span><br></pre></td></tr></table></figure>

<p>prometheus-rules中的规则字段可能随着版本更新出现变化，如有变化可以通知我，我实时更新文档。目前规则内的字段在此版本我已更新过。放心使用</p>
<p>还有一个小细节：prmetheus跟alertmanager的configmap是支持热更新的。可以用以下命令来热更新，可能执行刷新的时候会有点儿久，等一下就好</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl -X POST http:&#x2F;&#x2F;ClusterIP:PORT&#x2F;-&#x2F;reload</span><br></pre></td></tr></table></figure>
<p>资源下载：<a target="_blank" rel="noopener" href="https://github.com/alexclownfish/k8s-monitor">https://github.com/alexclownfish/k8s-monitor</a></p>
<h2 id="部署正文"><a href="#部署正文" class="headerlink" title="部署正文"></a>部署正文</h2><h3 id="创建ops命名空间"><a href="#创建ops命名空间" class="headerlink" title="创建ops命名空间"></a>创建ops命名空间</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create ns ops</span><br></pre></td></tr></table></figure>
<h3 id="prometheus-yaml文件"><a href="#prometheus-yaml文件" class="headerlink" title="prometheus yaml文件"></a>prometheus yaml文件</h3><h4 id="prometheus配置文件-prometheus-configmap-yaml"><a href="#prometheus配置文件-prometheus-configmap-yaml" class="headerlink" title="prometheus配置文件 prometheus-configmap.yaml"></a>prometheus配置文件 prometheus-configmap.yaml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus-config</span><br><span class="line">  namespace: ops </span><br><span class="line">data:</span><br><span class="line">  prometheus.yml: |</span><br><span class="line">    rule_files:</span><br><span class="line">    - &#x2F;etc&#x2F;config&#x2F;rules&#x2F;*.rules</span><br><span class="line"></span><br><span class="line">    scrape_configs:</span><br><span class="line">    - job_name: prometheus</span><br><span class="line">      static_configs:</span><br><span class="line">      - targets:</span><br><span class="line">        - localhost:9090</span><br><span class="line"></span><br><span class="line">    - job_name: kubernetes-apiservers</span><br><span class="line">      kubernetes_sd_configs:</span><br><span class="line">      - role: endpoints</span><br><span class="line">      relabel_configs:</span><br><span class="line">      - action: keep</span><br><span class="line">        regex: default;kubernetes;https</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_namespace</span><br><span class="line">        - __meta_kubernetes_service_name</span><br><span class="line">        - __meta_kubernetes_endpoint_port_name</span><br><span class="line">      scheme: https</span><br><span class="line">      tls_config:</span><br><span class="line">        ca_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;ca.crt</span><br><span class="line">        insecure_skip_verify: true</span><br><span class="line">      bearer_token_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;token</span><br><span class="line"> </span><br><span class="line">    - job_name: kubernetes-nodes-kubelet</span><br><span class="line">      kubernetes_sd_configs:</span><br><span class="line">      - role: node  # 发现集群中的节点</span><br><span class="line">      relabel_configs:</span><br><span class="line">      # 将标签(.*)作为新标签名，原有值不变</span><br><span class="line">      - action: labelmap</span><br><span class="line">        regex: __meta_kubernetes_node_label_(.+)</span><br><span class="line">      scheme: https</span><br><span class="line">      tls_config:</span><br><span class="line">        ca_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;ca.crt</span><br><span class="line">        insecure_skip_verify: true</span><br><span class="line">      bearer_token_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;token</span><br><span class="line"></span><br><span class="line">    - job_name: kubernetes-nodes-cadvisor</span><br><span class="line">      kubernetes_sd_configs:</span><br><span class="line">      - role: node</span><br><span class="line">      relabel_configs:</span><br><span class="line">      # 将标签(.*)作为新标签名，原有值不变</span><br><span class="line">      - action: labelmap</span><br><span class="line">        regex: __meta_kubernetes_node_label_(.+)</span><br><span class="line">      # 实际访问指标接口 https:&#x2F;&#x2F;NodeIP:10250&#x2F;metrics&#x2F;cadvisor，这里替换默认指标URL路径</span><br><span class="line">      - target_label: __metrics_path__</span><br><span class="line">        replacement: &#x2F;metrics&#x2F;cadvisor</span><br><span class="line">      scheme: https</span><br><span class="line">      tls_config:</span><br><span class="line">        ca_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;ca.crt</span><br><span class="line">        insecure_skip_verify: true</span><br><span class="line">      bearer_token_file: &#x2F;var&#x2F;run&#x2F;secrets&#x2F;kubernetes.io&#x2F;serviceaccount&#x2F;token</span><br><span class="line"></span><br><span class="line">    - job_name: kubernetes-service-endpoints</span><br><span class="line">      kubernetes_sd_configs:</span><br><span class="line">      - role: endpoints  # 从Service列表中的Endpoint发现Pod为目标</span><br><span class="line">      relabel_configs:</span><br><span class="line">      # Service没配置注解prometheus.io&#x2F;scrape的不采集</span><br><span class="line">      - action: keep</span><br><span class="line">        regex: true</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_service_annotation_prometheus_io_scrape</span><br><span class="line">      # 重命名采集目标协议</span><br><span class="line">      - action: replace</span><br><span class="line">        regex: (https?)</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_service_annotation_prometheus_io_scheme</span><br><span class="line">        target_label: __scheme__</span><br><span class="line">      # 重命名采集目标指标URL路径</span><br><span class="line">      - action: replace</span><br><span class="line">        regex: (.+)</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_service_annotation_prometheus_io_path</span><br><span class="line">        target_label: __metrics_path__</span><br><span class="line">      # 重命名采集目标地址</span><br><span class="line">      - action: replace</span><br><span class="line">        regex: ([^:]+)(?::\d+)?;(\d+)</span><br><span class="line">        replacement: $1:$2</span><br><span class="line">        source_labels:</span><br><span class="line">        - __address__</span><br><span class="line">        - __meta_kubernetes_service_annotation_prometheus_io_port</span><br><span class="line">        target_label: __address__</span><br><span class="line">      # 将K8s标签(.*)作为新标签名，原有值不变</span><br><span class="line">      - action: labelmap</span><br><span class="line">        regex: __meta_kubernetes_service_label_(.+)</span><br><span class="line">      # 生成命名空间标签</span><br><span class="line">      - action: replace</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_namespace</span><br><span class="line">        target_label: kubernetes_namespace</span><br><span class="line">      # 生成Service名称标签</span><br><span class="line">      - action: replace</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_service_name</span><br><span class="line">        target_label: kubernetes_name</span><br><span class="line"></span><br><span class="line">    - job_name: kubernetes-pods</span><br><span class="line">      kubernetes_sd_configs:</span><br><span class="line">      - role: pod   # 发现所有Pod为目标</span><br><span class="line">      # 重命名采集目标协议</span><br><span class="line">      relabel_configs:</span><br><span class="line">      - action: keep</span><br><span class="line">        regex: true</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_pod_annotation_prometheus_io_scrape</span><br><span class="line">      # 重命名采集目标指标URL路径</span><br><span class="line">      - action: replace</span><br><span class="line">        regex: (.+)</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_pod_annotation_prometheus_io_path</span><br><span class="line">        target_label: __metrics_path__</span><br><span class="line">      # 重命名采集目标地址</span><br><span class="line">      - action: replace</span><br><span class="line">        regex: ([^:]+)(?::\d+)?;(\d+)</span><br><span class="line">        replacement: $1:$2</span><br><span class="line">        source_labels:</span><br><span class="line">        - __address__</span><br><span class="line">        - __meta_kubernetes_pod_annotation_prometheus_io_port</span><br><span class="line">        target_label: __address__</span><br><span class="line">      # 将K8s标签(.*)作为新标签名，原有值不变</span><br><span class="line">      - action: labelmap</span><br><span class="line">        regex: __meta_kubernetes_pod_label_(.+)</span><br><span class="line">      # 生成命名空间标签</span><br><span class="line">      - action: replace</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_namespace</span><br><span class="line">        target_label: kubernetes_namespace</span><br><span class="line">      # 生成Service名称标签</span><br><span class="line">      - action: replace</span><br><span class="line">        source_labels:</span><br><span class="line">        - __meta_kubernetes_pod_name</span><br><span class="line">        target_label: kubernetes_pod_name</span><br><span class="line"></span><br><span class="line">    alerting:</span><br><span class="line">      alertmanagers:</span><br><span class="line">      - static_configs:</span><br><span class="line">          - targets: [&quot;alertmanager:80&quot;]</span><br></pre></td></tr></table></figure>
<h4 id="kube-state-metrics-采集了k8s中各种资源对象的状态信息-kube-state-metrics-yaml"><a href="#kube-state-metrics-采集了k8s中各种资源对象的状态信息-kube-state-metrics-yaml" class="headerlink" title="kube-state-metrics 采集了k8s中各种资源对象的状态信息 kube-state-metrics.yaml"></a>kube-state-metrics 采集了k8s中各种资源对象的状态信息 kube-state-metrics.yaml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1 </span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br><span class="line">  labels:</span><br><span class="line">    k8s-app: kube-state-metrics</span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      k8s-app: kube-state-metrics</span><br><span class="line">      version: v1.3.0</span><br><span class="line">  replicas: 1</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      labels:</span><br><span class="line">        k8s-app: kube-state-metrics</span><br><span class="line">        version: v1.3.0</span><br><span class="line">    spec:</span><br><span class="line">      serviceAccountName: kube-state-metrics</span><br><span class="line">      containers:</span><br><span class="line">      - name: kube-state-metrics</span><br><span class="line">        image: lizhenliang&#x2F;kube-state-metrics:v1.8.0 </span><br><span class="line">        ports:</span><br><span class="line">        - name: http-metrics</span><br><span class="line">          containerPort: 8080</span><br><span class="line">        - name: telemetry</span><br><span class="line">          containerPort: 8081</span><br><span class="line">        readinessProbe:</span><br><span class="line">          httpGet:</span><br><span class="line">            path: &#x2F;healthz</span><br><span class="line">            port: 8080</span><br><span class="line">          initialDelaySeconds: 5</span><br><span class="line">          timeoutSeconds: 5</span><br><span class="line">      - name: addon-resizer</span><br><span class="line">        image: lizhenliang&#x2F;addon-resizer:1.8.6</span><br><span class="line">        resources:</span><br><span class="line">          limits:</span><br><span class="line">            cpu: 100m</span><br><span class="line">            memory: 30Mi</span><br><span class="line">          requests:</span><br><span class="line">            cpu: 100m</span><br><span class="line">            memory: 30Mi</span><br><span class="line">        env:</span><br><span class="line">          - name: MY_POD_NAME</span><br><span class="line">            valueFrom:</span><br><span class="line">              fieldRef:</span><br><span class="line">                fieldPath: metadata.name</span><br><span class="line">          - name: MY_POD_NAMESPACE</span><br><span class="line">            valueFrom:</span><br><span class="line">              fieldRef:</span><br><span class="line">                fieldPath: metadata.namespace</span><br><span class="line">        volumeMounts:</span><br><span class="line">          - name: config-volume</span><br><span class="line">            mountPath: &#x2F;etc&#x2F;config</span><br><span class="line">        command:</span><br><span class="line">          - &#x2F;pod_nanny</span><br><span class="line">          - --config-dir&#x3D;&#x2F;etc&#x2F;config</span><br><span class="line">          - --container&#x3D;kube-state-metrics</span><br><span class="line">          - --cpu&#x3D;100m</span><br><span class="line">          - --extra-cpu&#x3D;1m</span><br><span class="line">          - --memory&#x3D;100Mi</span><br><span class="line">          - --extra-memory&#x3D;2Mi</span><br><span class="line">          - --threshold&#x3D;5</span><br><span class="line">          - --deployment&#x3D;kube-state-metrics</span><br><span class="line">      volumes:</span><br><span class="line">        - name: config-volume</span><br><span class="line">          configMap:</span><br><span class="line">            name: kube-state-metrics-config</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics-config</span><br><span class="line">  namespace: ops</span><br><span class="line">data:</span><br><span class="line">  NannyConfiguration: |-</span><br><span class="line">    apiVersion: nannyconfig&#x2F;v1alpha1</span><br><span class="line">    kind: NannyConfiguration</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br><span class="line">  annotations:</span><br><span class="line">    prometheus.io&#x2F;scrape: &#39;true&#39;</span><br><span class="line">spec:</span><br><span class="line">  ports:</span><br><span class="line">  - name: http-metrics</span><br><span class="line">    port: 8080</span><br><span class="line">    targetPort: http-metrics</span><br><span class="line">    protocol: TCP</span><br><span class="line">  - name: telemetry</span><br><span class="line">    port: 8081</span><br><span class="line">    targetPort: telemetry</span><br><span class="line">    protocol: TCP</span><br><span class="line">  selector:</span><br><span class="line">    k8s-app: kube-state-metrics</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ServiceAccount</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: ClusterRole</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">rules:</span><br><span class="line">- apiGroups: [&quot;&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - configmaps</span><br><span class="line">  - secrets</span><br><span class="line">  - nodes</span><br><span class="line">  - pods</span><br><span class="line">  - services</span><br><span class="line">  - resourcequotas</span><br><span class="line">  - replicationcontrollers</span><br><span class="line">  - limitranges</span><br><span class="line">  - persistentvolumeclaims</span><br><span class="line">  - persistentvolumes</span><br><span class="line">  - namespaces</span><br><span class="line">  - endpoints</span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;apps&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - statefulsets</span><br><span class="line">  - daemonsets</span><br><span class="line">  - deployments</span><br><span class="line">  - replicasets</span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;batch&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - cronjobs</span><br><span class="line">  - jobs</span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;autoscaling&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - horizontalpodautoscalers</span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;networking.k8s.io&quot;, &quot;extensions&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - ingresses </span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;storage.k8s.io&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - storageclasses </span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;certificates.k8s.io&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - certificatesigningrequests</span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line">- apiGroups: [&quot;policy&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - poddisruptionbudgets </span><br><span class="line">  verbs: [&quot;list&quot;, &quot;watch&quot;]</span><br><span class="line"></span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: Role</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics-resizer</span><br><span class="line">  namespace: ops</span><br><span class="line">rules:</span><br><span class="line">- apiGroups: [&quot;&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - pods</span><br><span class="line">  verbs: [&quot;get&quot;]</span><br><span class="line">- apiGroups: [&quot;extensions&quot;,&quot;apps&quot;]</span><br><span class="line">  resources:</span><br><span class="line">  - deployments</span><br><span class="line">  resourceNames: [&quot;kube-state-metrics&quot;]</span><br><span class="line">  verbs: [&quot;get&quot;, &quot;update&quot;]</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1 </span><br><span class="line">kind: ClusterRoleBinding</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">roleRef:</span><br><span class="line">  apiGroup: rbac.authorization.k8s.io</span><br><span class="line">  kind: ClusterRole</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">subjects:</span><br><span class="line">- kind: ServiceAccount</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: RoleBinding</span><br><span class="line">metadata:</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br><span class="line">roleRef:</span><br><span class="line">  apiGroup: rbac.authorization.k8s.io</span><br><span class="line">  kind: Role</span><br><span class="line">  name: kube-state-metrics-resizer</span><br><span class="line">subjects:</span><br><span class="line">- kind: ServiceAccount</span><br><span class="line">  name: kube-state-metrics</span><br><span class="line">  namespace: ops</span><br></pre></td></tr></table></figure>
<h4 id="prometheus部署文件-prometheus-deploy-yaml-注意版本需要用2-20"><a href="#prometheus部署文件-prometheus-deploy-yaml-注意版本需要用2-20" class="headerlink" title="prometheus部署文件  prometheus-deploy.yaml(注意版本需要用2.20)"></a>prometheus部署文件  prometheus-deploy.yaml(注意版本需要用2.20)</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus </span><br><span class="line">  namespace: ops</span><br><span class="line">  labels:</span><br><span class="line">    k8s-app: prometheus</span><br><span class="line">spec:</span><br><span class="line">  replicas: 1</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      k8s-app: prometheus</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      labels:</span><br><span class="line">        k8s-app: prometheus</span><br><span class="line">    spec:</span><br><span class="line">      serviceAccountName: prometheus</span><br><span class="line">      initContainers:</span><br><span class="line">      - name: &quot;init-chown-data&quot;</span><br><span class="line">        image: &quot;busybox:latest&quot;</span><br><span class="line">        imagePullPolicy: &quot;IfNotPresent&quot;</span><br><span class="line">        command: [&quot;chown&quot;, &quot;-R&quot;, &quot;65534:65534&quot;, &quot;&#x2F;data&quot;]</span><br><span class="line">        volumeMounts:</span><br><span class="line">        - name: prometheus-data</span><br><span class="line">          mountPath: &#x2F;data</span><br><span class="line">          subPath: &quot;&quot;</span><br><span class="line">      containers:</span><br><span class="line">        - name: prometheus-server-configmap-reload</span><br><span class="line">          image: &quot;jimmidyson&#x2F;configmap-reload:v0.1&quot;</span><br><span class="line">          imagePullPolicy: &quot;IfNotPresent&quot;</span><br><span class="line">          args:</span><br><span class="line">            - --volume-dir&#x3D;&#x2F;etc&#x2F;config</span><br><span class="line">            - --webhook-url&#x3D;http:&#x2F;&#x2F;localhost:9090&#x2F;-&#x2F;reload</span><br><span class="line">          volumeMounts:</span><br><span class="line">            - name: config-volume</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config</span><br><span class="line">              readOnly: true</span><br><span class="line">            - mountPath: &#x2F;etc&#x2F;localtime</span><br><span class="line">              name: timezone</span><br><span class="line">          resources:</span><br><span class="line">            limits:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 100Mi</span><br><span class="line">            requests:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 100Mi</span><br><span class="line"></span><br><span class="line">        - name: prometheus-server</span><br><span class="line">          image: &quot;prom&#x2F;prometheus:v2.20.0&quot;</span><br><span class="line">          imagePullPolicy: &quot;IfNotPresent&quot;</span><br><span class="line">          args:</span><br><span class="line">            - --config.file&#x3D;&#x2F;etc&#x2F;config&#x2F;prometheus.yml</span><br><span class="line">            - --storage.tsdb.path&#x3D;&#x2F;data</span><br><span class="line">            - --web.console.libraries&#x3D;&#x2F;etc&#x2F;prometheus&#x2F;console_libraries</span><br><span class="line">            - --web.console.templates&#x3D;&#x2F;etc&#x2F;prometheus&#x2F;consoles</span><br><span class="line">            - --web.enable-lifecycle</span><br><span class="line">          ports:</span><br><span class="line">            - containerPort: 9090</span><br><span class="line">          readinessProbe:</span><br><span class="line">            httpGet:</span><br><span class="line">              path: &#x2F;-&#x2F;ready</span><br><span class="line">              port: 9090</span><br><span class="line">            initialDelaySeconds: 30</span><br><span class="line">            timeoutSeconds: 30</span><br><span class="line">          livenessProbe:</span><br><span class="line">            httpGet:</span><br><span class="line">              path: &#x2F;-&#x2F;healthy</span><br><span class="line">              port: 9090</span><br><span class="line">            initialDelaySeconds: 30</span><br><span class="line">            timeoutSeconds: 30</span><br><span class="line">          resources:</span><br><span class="line">            limits:</span><br><span class="line">              cpu: 500m</span><br><span class="line">              memory: 800Mi</span><br><span class="line">            requests:</span><br><span class="line">              cpu: 200m</span><br><span class="line">              memory: 400Mi</span><br><span class="line">            </span><br><span class="line">          volumeMounts:</span><br><span class="line">            - name: config-volume</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config</span><br><span class="line">            - name: prometheus-data</span><br><span class="line">              mountPath: &#x2F;data</span><br><span class="line">              subPath: &quot;&quot;</span><br><span class="line">            - name: prometheus-rules</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config&#x2F;rules</span><br><span class="line">            - mountPath: &#x2F;etc&#x2F;localtime</span><br><span class="line">              name: timezone  </span><br><span class="line">      volumes:</span><br><span class="line">        - name: config-volume</span><br><span class="line">          configMap:</span><br><span class="line">            name: prometheus-config</span><br><span class="line">        - name: prometheus-rules</span><br><span class="line">          configMap:</span><br><span class="line">            name: prometheus-rules</span><br><span class="line">        - name: prometheus-data</span><br><span class="line">          persistentVolumeClaim:</span><br><span class="line">            claimName: prometheus</span><br><span class="line">        - name: timezone</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;usr&#x2F;share&#x2F;zoneinfo&#x2F;Asia&#x2F;Shanghai</span><br><span class="line">                                                  </span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus</span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: &quot;nfs-storage&quot;</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteMany</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 10Gi</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata: </span><br><span class="line">  name: prometheus</span><br><span class="line">  namespace: ops</span><br><span class="line">spec: </span><br><span class="line">  type: NodePort</span><br><span class="line">  ports: </span><br><span class="line">    - name: http </span><br><span class="line">      port: 9090</span><br><span class="line">      protocol: TCP</span><br><span class="line">      targetPort: 9090</span><br><span class="line">      nodePort: 30089</span><br><span class="line">  selector: </span><br><span class="line">    k8s-app: prometheus</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ServiceAccount</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus</span><br><span class="line">  namespace: ops</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: ClusterRole</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus</span><br><span class="line">rules:</span><br><span class="line">  - apiGroups:</span><br><span class="line">      - &quot;&quot;</span><br><span class="line">    resources:</span><br><span class="line">      - nodes</span><br><span class="line">      - nodes&#x2F;metrics</span><br><span class="line">      - services</span><br><span class="line">      - endpoints</span><br><span class="line">      - pods</span><br><span class="line">    verbs:</span><br><span class="line">      - get</span><br><span class="line">      - list</span><br><span class="line">      - watch</span><br><span class="line">  - apiGroups:</span><br><span class="line">      - &quot;&quot;</span><br><span class="line">    resources:</span><br><span class="line">      - configmaps</span><br><span class="line">    verbs:</span><br><span class="line">      - get</span><br><span class="line">  - nonResourceURLs:</span><br><span class="line">      - &quot;&#x2F;metrics&quot;</span><br><span class="line">    verbs:</span><br><span class="line">      - get</span><br><span class="line">---</span><br><span class="line">apiVersion: rbac.authorization.k8s.io&#x2F;v1</span><br><span class="line">kind: ClusterRoleBinding</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus</span><br><span class="line">roleRef:</span><br><span class="line">  apiGroup: rbac.authorization.k8s.io</span><br><span class="line">  kind: ClusterRole</span><br><span class="line">  name: prometheus</span><br><span class="line">subjects:</span><br><span class="line">- kind: ServiceAccount</span><br><span class="line">  name: prometheus</span><br><span class="line">  namespace: ops  </span><br></pre></td></tr></table></figure>
<h4 id="prometheus配置报警规则-prometheus-rules-yaml"><a href="#prometheus配置报警规则-prometheus-rules-yaml" class="headerlink" title="prometheus配置报警规则 prometheus-rules.yaml"></a>prometheus配置报警规则 prometheus-rules.yaml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">metadata:</span><br><span class="line">  name: prometheus-rules</span><br><span class="line">  namespace: ops</span><br><span class="line">data:</span><br><span class="line">  general.rules: |</span><br><span class="line">    groups:</span><br><span class="line">    - name: general.rules</span><br><span class="line">      rules:</span><br><span class="line">      - alert: InstanceDown</span><br><span class="line">        expr: up &#x3D;&#x3D; 0</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: error </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Instance &#123;&#123; $labels.instance &#125;&#125; 停止工作&quot;</span><br><span class="line">          description: &quot;&#123;&#123; $labels.instance &#125;&#125; job &#123;&#123; $labels.job &#125;&#125; 已经停止5分钟以上.&quot;</span><br><span class="line">               </span><br><span class="line">  node.rules: |</span><br><span class="line">    groups:</span><br><span class="line">    - name: node.rules</span><br><span class="line">      rules:</span><br><span class="line">      - alert: NodeFilesystemUsage</span><br><span class="line">        expr: |</span><br><span class="line">          100 - (node_filesystem_free_bytes &#x2F; node_filesystem_size_bytes) * 100 &gt; 60</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Instance &#123;&#123; $labels.instance &#125;&#125; : &#123;&#123; $labels.mountpoint &#125;&#125; 分区使用率过高&quot;</span><br><span class="line">          description: &quot;&#123;&#123; $labels.instance &#125;&#125;: &#123;&#123; $labels.mountpoint &#125;&#125; 分区使用大于60% (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: NodeMemoryUsage</span><br><span class="line">        expr: |</span><br><span class="line">          100 - (node_memory_MemFree_bytes+node_memory_Cached_bytes+node_memory_Buffers_bytes) &#x2F; node_memory_MemTotal_bytes * 100 &gt; 60</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Instance &#123;&#123; $labels.instance &#125;&#125; 内存使用率过高&quot;</span><br><span class="line">          description: &quot;&#123;&#123; $labels.instance &#125;&#125;内存使用大于60% (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: NodeCPUUsage    </span><br><span class="line">        expr: |</span><br><span class="line">          100 - (avg(irate(node_cpu_seconds_total&#123;mode&#x3D;&quot;idle&quot;&#125;[5m])) by (instance) * 100) &gt; 60 </span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Instance &#123;&#123; $labels.instance &#125;&#125; CPU使用率过高&quot;       </span><br><span class="line">          description: &quot;&#123;&#123; $labels.instance &#125;&#125;CPU使用大于60% (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: KubeNodeNotReady</span><br><span class="line">        expr: |</span><br><span class="line">          kube_node_status_condition&#123;condition&#x3D;&quot;Ready&quot;,status&#x3D;&quot;true&quot;&#125; &#x3D;&#x3D; 0</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: error</span><br><span class="line">        annotations:</span><br><span class="line">          message: &#39;&#123;&#123; $labels.node &#125;&#125; 已经有10多分钟没有准备好了.&#39;</span><br><span class="line"></span><br><span class="line">  pod.rules: |</span><br><span class="line">    groups:</span><br><span class="line">    - name: pod.rules</span><br><span class="line">      rules:</span><br><span class="line">      - alert: PodCPUUsage</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(pod, namespace) (rate(container_cpu_usage_seconds_total&#123;image!&#x3D;&quot;&quot;&#125;[5m]) * 100) &gt; 5</span><br><span class="line">        for: 5m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; CPU使用大于80% (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodMemoryUsage</span><br><span class="line">        expr: |</span><br><span class="line">           sum(container_memory_rss&#123;image!&#x3D;&quot;&quot;&#125;) by(pod, namespace) &#x2F; sum(container_spec_memory_limit_bytes&#123;image!&#x3D;&quot;&quot;&#125;) by(pod, namespace) * 100 !&#x3D; +inf &gt; 80</span><br><span class="line">        for: 5m</span><br><span class="line">        labels:</span><br><span class="line">          severity: error </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; 内存使用大于80% (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodNetworkReceive</span><br><span class="line">        expr: |</span><br><span class="line">           sum(rate(container_network_receive_bytes_total&#123;image!&#x3D;&quot;&quot;,name&#x3D;~&quot;^k8s_.*&quot;&#125;[5m]) &#x2F;1000) by (pod,namespace) &gt; 30000</span><br><span class="line">        for: 5m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; 入口流量大于30MB&#x2F;s (当前值: &#123;&#123; $value &#125;&#125;K&#x2F;s)&quot;           </span><br><span class="line"></span><br><span class="line">      - alert: PodNetworkTransmit</span><br><span class="line">        expr: | </span><br><span class="line">           sum(rate(container_network_transmit_bytes_total&#123;image!&#x3D;&quot;&quot;,name&#x3D;~&quot;^k8s_.*&quot;&#125;[5m]) &#x2F;1000) by (pod,namespace) &gt; 30000</span><br><span class="line">        for: 5m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; 出口流量大于30MB&#x2F;s (当前值: &#123;&#123; $value &#125;&#125;&#x2F;K&#x2F;s)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodRestart</span><br><span class="line">        expr: |</span><br><span class="line">           sum(changes(kube_pod_container_status_restarts_total[1m])) by (pod,namespace) &gt; 0</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; Pod重启 (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodFailed</span><br><span class="line">        expr: |</span><br><span class="line">           sum(kube_pod_status_phase&#123;phase&#x3D;&quot;Failed&quot;&#125;) by (pod,namespace) &gt; 0</span><br><span class="line">        for: 5s</span><br><span class="line">        labels:</span><br><span class="line">          severity: error </span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; Pod状态Failed (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodPending</span><br><span class="line">        expr: | </span><br><span class="line">           sum(kube_pod_status_phase&#123;phase&#x3D;&quot;Pending&quot;&#125;) by (pod,namespace) &gt; 0</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: error</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125; Pod状态Pending (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodErrImagePull</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace,pod) (kube_pod_container_status_waiting_reason&#123;reason&#x3D;&quot;ErrImagePull&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125;  Pod状态ErrImagePull (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodImagePullBackOff</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace,pod) (kube_pod_container_status_waiting_reason&#123;reason&#x3D;&quot;ImagePullBackOff&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125;  Pod状态ImagePullBackOff (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodCrashLoopBackOff</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace,pod) (kube_pod_container_status_waiting_reason&#123;reason&#x3D;&quot;CrashLoopBackOff&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125;  Pod状态CrashLoopBackOff (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodInvalidImageName</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace,pod) (kube_pod_container_status_waiting_reason&#123;reason&#x3D;&quot;InvalidImageName&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125;  Pod状态InvalidImageName (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">      - alert: PodCreateContainerConfigError</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace,pod) (kube_pod_container_status_waiting_reason&#123;reason&#x3D;&quot;CreateContainerConfigError&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 1m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;命名空间: &#123;&#123; $labels.namespace &#125;&#125; | Pod名称: &#123;&#123; $labels.pod &#125;&#125;  Pod状态CreateContainerConfigError (当前值: &#123;&#123; $value &#125;&#125;)&quot;</span><br><span class="line"></span><br><span class="line">  volume.rules: |</span><br><span class="line">    groups:</span><br><span class="line">    - name: volume.rules</span><br><span class="line">      rules:</span><br><span class="line">      - alert: PersistentVolumeClaimLost</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase&#123;phase&#x3D;&quot;Lost&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 2m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;PersistentVolumeClaim &#123;&#123; $labels.namespace &#125;&#125;&#x2F;&#123;&#123; $labels.persistentvolumeclaim &#125;&#125; is lost\n  VALUE &#x3D; &#123;&#123; $value &#125;&#125;\n  LABELS &#x3D; &#123;&#123; $labels &#125;&#125;&quot;</span><br><span class="line">      - alert: PersistentVolumeClaimPendig</span><br><span class="line">        expr: |</span><br><span class="line">           sum by(namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase&#123;phase&#x3D;&quot;Pendig&quot;&#125;) &#x3D;&#x3D; 1</span><br><span class="line">        for: 2m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;PersistentVolumeClaim &#123;&#123; $labels.namespace &#125;&#125;&#x2F;&#123;&#123; $labels.persistentvolumeclaim &#125;&#125; is pendig\n  VALUE &#x3D; &#123;&#123; $value &#125;&#125;\n  LABELS &#x3D; &#123;&#123; $labels &#125;&#125;&quot;</span><br><span class="line">      - alert: PersistentVolume Failed</span><br><span class="line">        expr: |</span><br><span class="line">           sum(kube_persistentvolume_status_phase&#123;phase&#x3D;&quot;Failed&quot;,job&#x3D;&quot;kubernetes-service-endpoints&quot;&#125;) by (persistentvolume) &#x3D;&#x3D; 1</span><br><span class="line">        for: 2m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Persistent volume is failed state\n  VALUE &#x3D; &#123;&#123; $value &#125;&#125;\n  LABELS &#x3D; &#123;&#123; $labels &#125;&#125;&quot;</span><br><span class="line">      - alert: PersistentVolume Pending</span><br><span class="line">        expr: |</span><br><span class="line">           sum(kube_persistentvolume_status_phase&#123;phase&#x3D;&quot;Pending&quot;,job&#x3D;&quot;kubernetes-service-endpoints&quot;&#125;) by (persistentvolume) &#x3D;&#x3D; 1</span><br><span class="line">        for: 2m</span><br><span class="line">        labels:</span><br><span class="line">          severity: warning</span><br><span class="line">        annotations:</span><br><span class="line">          summary: &quot;Persistent volume is pending state\n  VALUE &#x3D; &#123;&#123; $value &#125;&#125;\n  LABELS &#x3D; &#123;&#123; $labels &#125;&#125;&quot;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h4 id="node-exporter配置node-exporter-yaml-注意版本需要用1-0-1"><a href="#node-exporter配置node-exporter-yaml-注意版本需要用1-0-1" class="headerlink" title="node-exporter配置node-exporter.yaml(注意版本需要用1.0.1)"></a>node-exporter配置node-exporter.yaml(注意版本需要用1.0.1)</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1 </span><br><span class="line">kind: DaemonSet</span><br><span class="line">metadata:</span><br><span class="line">  name: node-exporter </span><br><span class="line">  namespace: ops </span><br><span class="line">  labels:</span><br><span class="line">    k8s-app: node-exporter </span><br><span class="line">spec:</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      k8s-app: node-exporter</span><br><span class="line">      version: v1.0.1</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      labels:</span><br><span class="line">        k8s-app: node-exporter </span><br><span class="line">        version: v1.0.1</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">        - name: prometheus-node-exporter</span><br><span class="line">          image: &quot;prom&#x2F;node-exporter:v1.0.1&quot;</span><br><span class="line">          #imagePullPolicy: &quot;Always&quot;</span><br><span class="line">          args:</span><br><span class="line">            - --path.procfs&#x3D;&#x2F;host&#x2F;proc</span><br><span class="line">            - --path.sysfs&#x3D;&#x2F;host&#x2F;sys</span><br><span class="line">          ports:</span><br><span class="line">            - name: metrics</span><br><span class="line">              containerPort: 9100</span><br><span class="line">              hostPort: 9100</span><br><span class="line">          volumeMounts:</span><br><span class="line">            - name: proc</span><br><span class="line">              mountPath: &#x2F;host&#x2F;proc</span><br><span class="line">              readOnly:  true</span><br><span class="line">            - name: sys</span><br><span class="line">              mountPath: &#x2F;host&#x2F;sys</span><br><span class="line">              readOnly: true</span><br><span class="line">          resources:</span><br><span class="line">            limits:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 50Mi</span><br><span class="line">            requests:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 50Mi</span><br><span class="line">      hostNetwork: true</span><br><span class="line">      hostPID: true</span><br><span class="line">      hostIPC: true</span><br><span class="line">      volumes:</span><br><span class="line">        - name: proc</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;proc</span><br><span class="line">        - name: sys</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;sys</span><br><span class="line">        - name: rootfs</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;</span><br><span class="line">        - name: dev</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;dev</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: node-exporter</span><br><span class="line">  namespace: ops</span><br><span class="line">  annotations:</span><br><span class="line">    prometheus.io&#x2F;scrape: &quot;true&quot;</span><br><span class="line">spec:</span><br><span class="line">  clusterIP: None</span><br><span class="line">  ports:</span><br><span class="line">    - name: metrics</span><br><span class="line">      port: 9100</span><br><span class="line">      protocol: TCP</span><br><span class="line">      targetPort: 9100</span><br><span class="line">  selector:</span><br><span class="line">    k8s-app: node-exporter</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h3 id="alertmanager-yaml文件"><a href="#alertmanager-yaml文件" class="headerlink" title="alertmanager yaml文件"></a>alertmanager yaml文件</h3><h4 id="alertmanager配置文件alertmanger-configmap-yaml"><a href="#alertmanager配置文件alertmanger-configmap-yaml" class="headerlink" title="alertmanager配置文件alertmanger-configmap.yaml"></a>alertmanager配置文件alertmanger-configmap.yaml</h4><p>注:邮箱需要自己去网易邮箱申请并且取得授权管理密码</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">metadata:</span><br><span class="line">  name: alertmanager-config</span><br><span class="line">  namespace: ops</span><br><span class="line">data:</span><br><span class="line">  alertmanager.yml: |-</span><br><span class="line">    global:</span><br><span class="line">      # 在没有报警的情况下声明为已解决的时间</span><br><span class="line">      resolve_timeout: 5m</span><br><span class="line">      # 配置邮件发送信息</span><br><span class="line">      smtp_smarthost: &#39;smtp.163.com:465&#39;</span><br><span class="line">      smtp_from: &#39;xxx@163.com&#39;</span><br><span class="line">      smtp_auth_username: &#39;xxx@163.com&#39;</span><br><span class="line">      smtp_auth_password: &#39;xxxxxx&#39;</span><br><span class="line">      smtp_hello: &#39;163.com&#39;</span><br><span class="line">      smtp_require_tls: false</span><br><span class="line">    # 所有报警信息进入后的根路由，用来设置报警的分发策略</span><br><span class="line">    route:</span><br><span class="line">      # 这里的标签列表是接收到报警信息后的重新分组标签，例如，接收到的报警信息里面有许多具有 cluster&#x3D;A 和 alertname&#x3D;LatncyHigh 这样的标签的报警信息将会批量被聚合到一个分组里面</span><br><span class="line">      group_by: [&#39;alertname&#39;, &#39;cluster&#39;]</span><br><span class="line">      # 当一个新的报警分组被创建后，需要等待至少group_wait时间来初始化通知，这种方式可以确保您能有足够的时间为同一分组来获取多个警报，然后一起触发这个报警信息。</span><br><span class="line">      group_wait: 30s</span><br><span class="line"> </span><br><span class="line">      # 当第一个报警发送后，等待&#39;group_interval&#39;时间来发送新的一组报警信息。</span><br><span class="line">      group_interval: 5m</span><br><span class="line"> </span><br><span class="line">      # 如果一个报警信息已经发送成功了，等待&#39;repeat_interval&#39;时间来重新发送他们</span><br><span class="line">      repeat_interval: 5m</span><br><span class="line"> </span><br><span class="line">      # 默认的receiver：如果一个报警没有被一个route匹配，则发送给默认的接收器</span><br><span class="line">      receiver: default</span><br><span class="line"> </span><br><span class="line">      # 上面所有的属性都由所有子路由继承，并且可以在每个子路由上进行覆盖。</span><br><span class="line">      routes:</span><br><span class="line">      - receiver: email</span><br><span class="line">        group_wait: 10s</span><br><span class="line">        match:</span><br><span class="line">          team: node</span><br><span class="line">    templates:</span><br><span class="line">      - &#39;&#x2F;etc&#x2F;config&#x2F;template&#x2F;email.tmpl&#39;</span><br><span class="line">    receivers:</span><br><span class="line">    - name: &#39;default&#39;</span><br><span class="line">      email_configs:</span><br><span class="line">      - to: &#39;xxxx@qq.com&#39;</span><br><span class="line">        html: &#39;&#123;&#123; template &quot;email.html&quot; . &#125;&#125;&#39;</span><br><span class="line">        headers: &#123; Subject: &quot;[WARN] Prometheus 告警邮件&quot; &#125;</span><br><span class="line">        #send_resolved: true</span><br><span class="line">    - name: &#39;email&#39;</span><br><span class="line">      email_configs:</span><br><span class="line">      - to: &#39;xxxx@gmail.com&#39;</span><br><span class="line">        send_resolved: true</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h4 id="alertmanager-template文件alertmanager-template-yaml"><a href="#alertmanager-template文件alertmanager-template-yaml" class="headerlink" title="alertmanager template文件alertmanager-template.yaml"></a>alertmanager template文件alertmanager-template.yaml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">#自定义告警模板</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">metadata:</span><br><span class="line">  name: alertmanager-template-volume</span><br><span class="line">  namespace: ops</span><br><span class="line">data:</span><br><span class="line">  email.tmpl: |</span><br><span class="line">    &#123;&#123; define &quot;email.html&quot; &#125;&#125;</span><br><span class="line">        &#123;&#123; range .Alerts &#125;&#125;</span><br><span class="line">    &lt;pre&gt;</span><br><span class="line">        &#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;start&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;</span><br><span class="line">       告警程序: prometheus_alert_email </span><br><span class="line">       告警级别: &#123;&#123; .Labels.severity &#125;&#125; 级别 </span><br><span class="line">       告警类型: &#123;&#123; .Labels.alertname &#125;&#125; </span><br><span class="line">       故障主机: &#123;&#123; .Labels.instance &#125;&#125; </span><br><span class="line">       告警主题: &#123;&#123; .Annotations.summary &#125;&#125;</span><br><span class="line">       告警详情: &#123;&#123; .Annotations.description &#125;&#125;</span><br><span class="line">       处理方法: &#123;&#123; .Annotations.console &#125;&#125;</span><br><span class="line">       触发时间: &#123;&#123; .StartsAt.Format &quot;2006-01-02 15:04:05&quot; &#125;&#125;</span><br><span class="line">       &#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;end&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;&#x3D;</span><br><span class="line">    &lt;&#x2F;pre&gt;</span><br><span class="line">        &#123;&#123; end &#125;&#125;</span><br><span class="line">    &#123;&#123; end &#125;&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h4 id="alertmanager部署文件alertmanager-deployment-yaml"><a href="#alertmanager部署文件alertmanager-deployment-yaml" class="headerlink" title="alertmanager部署文件alertmanager-deployment.yaml"></a>alertmanager部署文件alertmanager-deployment.yaml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  name: alertmanager</span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  replicas: 1</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      k8s-app: alertmanager</span><br><span class="line">      version: v0.14.0</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      labels:</span><br><span class="line">        k8s-app: alertmanager</span><br><span class="line">        version: v0.14.0</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">        - name: prometheus-alertmanager</span><br><span class="line">          image: &quot;prom&#x2F;alertmanager:v0.14.0&quot;</span><br><span class="line">          imagePullPolicy: &quot;IfNotPresent&quot;</span><br><span class="line">          args:</span><br><span class="line">            - --config.file&#x3D;&#x2F;etc&#x2F;config&#x2F;alertmanager.yml</span><br><span class="line">            - --storage.path&#x3D;&#x2F;data</span><br><span class="line">            - --web.external-url&#x3D;&#x2F;</span><br><span class="line">          ports:</span><br><span class="line">            - containerPort: 9093</span><br><span class="line">          readinessProbe:</span><br><span class="line">            httpGet:</span><br><span class="line">              path: &#x2F;#&#x2F;status</span><br><span class="line">              port: 9093</span><br><span class="line">            initialDelaySeconds: 30</span><br><span class="line">            timeoutSeconds: 30</span><br><span class="line">          volumeMounts:</span><br><span class="line">            - name: config-volume</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config</span><br><span class="line">#自定义告警模板</span><br><span class="line">            - name: config-template-volume</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config&#x2F;template</span><br><span class="line">            - name: storage-volume</span><br><span class="line">              mountPath: &quot;&#x2F;data&quot;</span><br><span class="line">              subPath: &quot;&quot;</span><br><span class="line">            - mountPath: &#x2F;etc&#x2F;localtime</span><br><span class="line">              name: timezone</span><br><span class="line">          resources:</span><br><span class="line">            limits:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 200Mi</span><br><span class="line">            requests:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 100Mi</span><br><span class="line">        - name: prometheus-alertmanager-configmap-reload</span><br><span class="line">          image: &quot;jimmidyson&#x2F;configmap-reload:v0.1&quot;</span><br><span class="line">          imagePullPolicy: &quot;IfNotPresent&quot;</span><br><span class="line">          args:</span><br><span class="line">            - --volume-dir&#x3D;&#x2F;etc&#x2F;config</span><br><span class="line">            - --webhook-url&#x3D;http:&#x2F;&#x2F;localhost:9093&#x2F;-&#x2F;reload</span><br><span class="line">          volumeMounts:</span><br><span class="line">            - name: config-volume</span><br><span class="line">              mountPath: &#x2F;etc&#x2F;config</span><br><span class="line">              readOnly: true</span><br><span class="line">          resources:</span><br><span class="line">            limits:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 200Mi</span><br><span class="line">            requests:</span><br><span class="line">              cpu: 10m</span><br><span class="line">              memory: 100Mi</span><br><span class="line">      volumes:</span><br><span class="line">        - name: config-volume</span><br><span class="line">          configMap:</span><br><span class="line">            name: alertmanager-config</span><br><span class="line">        - name: config-template-volume</span><br><span class="line">          configMap:</span><br><span class="line">            name: alertmanager-template-volume</span><br><span class="line">        - name: storage-volume</span><br><span class="line">          persistentVolumeClaim:</span><br><span class="line">            claimName: alertmanager</span><br><span class="line">        - name: timezone</span><br><span class="line">          hostPath:</span><br><span class="line">            path: &#x2F;usr&#x2F;share&#x2F;zoneinfo&#x2F;Asia&#x2F;Shanghai</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: alertmanager</span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: nfs-storage</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteOnce</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: &quot;2Gi&quot;</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: alertmanager</span><br><span class="line">  namespace: ops</span><br><span class="line">  labels:</span><br><span class="line">    kubernetes.io&#x2F;cluster-service: &quot;true&quot;</span><br><span class="line">    addonmanager.kubernetes.io&#x2F;mode: Reconcile</span><br><span class="line">    kubernetes.io&#x2F;name: &quot;Alertmanager&quot;</span><br><span class="line">spec:</span><br><span class="line">  type: &quot;NodePort&quot;</span><br><span class="line">  ports:</span><br><span class="line">    - name: http</span><br><span class="line">      port: 80</span><br><span class="line">      protocol: TCP</span><br><span class="line">      targetPort: 9093</span><br><span class="line">      nodePort: 30093</span><br><span class="line">  selector:</span><br><span class="line">    k8s-app: alertmanager</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h3 id="grafana-yaml文件"><a href="#grafana-yaml文件" class="headerlink" title="grafana yaml文件"></a>grafana yaml文件</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment </span><br><span class="line">metadata:</span><br><span class="line">  name: grafana</span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  replicas: 1</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      app: grafana</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      labels:</span><br><span class="line">        app: grafana</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">      - name: grafana</span><br><span class="line">        image: grafana&#x2F;grafana:7.1.0</span><br><span class="line">        ports:</span><br><span class="line">          - containerPort: 3000</span><br><span class="line">            protocol: TCP</span><br><span class="line">        resources:</span><br><span class="line">          limits:</span><br><span class="line">            cpu: 100m            </span><br><span class="line">            memory: 256Mi          </span><br><span class="line">          requests:</span><br><span class="line">            cpu: 100m            </span><br><span class="line">            memory: 256Mi</span><br><span class="line">        volumeMounts:</span><br><span class="line">          - name: grafana-data</span><br><span class="line">            mountPath: &#x2F;var&#x2F;lib&#x2F;grafana</span><br><span class="line">            subPath: grafana</span><br><span class="line">          - mountPath: &#x2F;etc&#x2F;localtime</span><br><span class="line">            name: timezone</span><br><span class="line">      securityContext:</span><br><span class="line">        fsGroup: 472</span><br><span class="line">        runAsUser: 472</span><br><span class="line">      volumes:</span><br><span class="line">      - name: grafana-data</span><br><span class="line">        persistentVolumeClaim:</span><br><span class="line">          claimName: grafana</span><br><span class="line">      - name: timezone</span><br><span class="line">        hostPath:</span><br><span class="line">          path: &#x2F;usr&#x2F;share&#x2F;zoneinfo&#x2F;Asia&#x2F;Shanghai </span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: grafana </span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: &quot;nfs-storage&quot;</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteMany</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 5Gi</span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Service</span><br><span class="line">metadata:</span><br><span class="line">  name: grafana</span><br><span class="line">  namespace: ops</span><br><span class="line">spec:</span><br><span class="line">  type: NodePort</span><br><span class="line">  ports:</span><br><span class="line">  - port : 80</span><br><span class="line">    targetPort: 3000</span><br><span class="line">    nodePort: 30030</span><br><span class="line">  selector:</span><br><span class="line">    app: grafana</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h3 id="部署到k8s中"><a href="#部署到k8s中" class="headerlink" title="部署到k8s中"></a>部署到k8s中</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f .</span><br></pre></td></tr></table></figure>

<h2 id="grafana数据源和监控"><a href="#grafana数据源和监控" class="headerlink" title="grafana数据源和监控"></a>grafana数据源和监控</h2><h3 id="grafana添加数据源"><a href="#grafana添加数据源" class="headerlink" title="grafana添加数据源"></a>grafana添加数据源</h3><p><img src="https://img-blog.csdnimg.cn/img_convert/2e2b52257bf58eb3de3800b73bf1b33a.png" alt="在这里插入图片描述"><br>点击datasource - add datasource<br><img src="https://img-blog.csdnimg.cn/img_convert/f4a6d0482662d940b54f5345ff5719b6.png" alt="在这里插入图片描述"><br>之后点击save&amp;test,添加数据源结束</p>
<h3 id="import导入模板"><a href="#import导入模板" class="headerlink" title="import导入模板"></a>import导入模板</h3><p>模板下载：<a target="_blank" rel="noopener" href="https://github.com/alexclownfish/k8s-monitor/tree/main/grafana_template">https://github.com/alexclownfish/k8s-monitor/tree/main/grafana_template</a><br><img src="https://img-blog.csdnimg.cn/img_convert/0da7161b13ec62e8a64bb700885473fc.png" alt="在这里插入图片描述"><br><img src="https://img-blog.csdnimg.cn/img_convert/6dc324132e15af124c96a6cb10632f20.png" alt="在这里插入图片描述"></p>
<h3 id="修改prometheus-rules验证监控触发报警并发送邮件"><a href="#修改prometheus-rules验证监控触发报警并发送邮件" class="headerlink" title="修改prometheus rules验证监控触发报警并发送邮件"></a>修改prometheus rules验证监控触发报警并发送邮件</h3><p>修改prometheus-rules.yaml<br><img src="https://img-blog.csdnimg.cn/img_convert/68786c7f9abf4518c5cf910056074369.png" alt="在这里插入图片描述"></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">#热更新configmap</span><br><span class="line">kubectl apply -f prometheus-rules.yaml</span><br><span class="line">curl -X POST http:&#x2F;&#x2F;10.1.230.219:9090&#x2F;-&#x2F;reload</span><br></pre></td></tr></table></figure>
<p><img src="https://img-blog.csdnimg.cn/img_convert/9bed866e7556554490febdfb84bf3769.png" alt="在这里插入图片描述"><br><img src="https://img-blog.csdnimg.cn/img_convert/7a5776cc75bbd830ee989699467317c4.png" alt="在这里插入图片描述"></p>
<p>看到已经触发报警并发送邮件<br>至此结束</p>
<h2 id="感谢大佬"><a href="#感谢大佬" class="headerlink" title="感谢大佬"></a>感谢大佬</h2><p><a target="_blank" rel="noopener" href="https://blog.51cto.com/luoguoling">https://blog.51cto.com/luoguoling</a><br><a href="https://alexcld.com/">https://alexcld.com</a></p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">Alex</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://alexcld.com/posts/34136.html">http://alexcld.com/posts/34136.html</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://alexcld.com" target="_blank">Alexcld</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/prometheus-grafana-alertmanager%E7%9B%91%E6%8E%A7k8s%E6%97%A0%E5%9D%91%E7%89%88/">prometheus+grafana+alertmanager监控k8s无坑版</a></div><div class="post_share"><div class="social-share" data-image="https://img.alexcld.com/img/214606-15553359668cc5.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/posts/24142.html"><img class="prev-cover" src="https://img.alexcld.com/img/193615-1568028975fe7b.jpg" onerror="onerror=null;src='/img/404.jpg'"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">Prometheus邮件告警自定义模板的触发时间为utc的解决方法</div></div></a></div><div class="next-post pull-right"><a href="/posts/35255.html"><img class="next-cover" src="https://img.alexcld.com/img/231712-160450303247b9.jpg" onerror="onerror=null;src='/img/404.jpg'"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">clickhouse升级最新版21.6.6</div></div></a></div></nav><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div><div id="comment-switch"><span class="first-comment">Valine</span><span class="switch-btn"></span><span class="second-comment">Twikoo</span></div></div><div class="comment-wrap"><div><div class="vcomment" id="vcomment"></div></div><div><div id="twikoo"></div></div></div></div></div><div class="aside_content" id="aside_content"><div class="card-widget card-info"><div class="card-content"><div class="card-info-avatar is-center"><img class="avatar-img" src="https://img.alexcld.com/img/author.png" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/><div class="author-info__name">Alex</div><div class="author-info__description">children don't lie</div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">24</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">24</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div></div><a class="button--animated" id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/alexclownfish"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/alexclownfish" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="mailto:ywz0207@163.com" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div></div><div class="card-widget card-announcement"><div class="card-content"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content">This is AKA Alex,respect yeah</div><timing></timing></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="card-content"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#prometheus-grafana-alertmanager%E7%9B%91%E6%8E%A7k8s%E6%97%A0%E5%9D%91%E7%89%88"><span class="toc-number">1.</span> <span class="toc-text">prometheus+grafana+alertmanager监控k8s无坑版</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%91%98%E8%A6%81"><span class="toc-number">1.1.</span> <span class="toc-text">摘要</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%83%A8%E7%BD%B2%E6%AD%A3%E6%96%87"><span class="toc-number">1.2.</span> <span class="toc-text">部署正文</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%9B%E5%BB%BAops%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4"><span class="toc-number">1.2.1.</span> <span class="toc-text">创建ops命名空间</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#prometheus-yaml%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.2.</span> <span class="toc-text">prometheus yaml文件</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#prometheus%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6-prometheus-configmap-yaml"><span class="toc-number">1.2.2.1.</span> <span class="toc-text">prometheus配置文件 prometheus-configmap.yaml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#kube-state-metrics-%E9%87%87%E9%9B%86%E4%BA%86k8s%E4%B8%AD%E5%90%84%E7%A7%8D%E8%B5%84%E6%BA%90%E5%AF%B9%E8%B1%A1%E7%9A%84%E7%8A%B6%E6%80%81%E4%BF%A1%E6%81%AF-kube-state-metrics-yaml"><span class="toc-number">1.2.2.2.</span> <span class="toc-text">kube-state-metrics 采集了k8s中各种资源对象的状态信息 kube-state-metrics.yaml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#prometheus%E9%83%A8%E7%BD%B2%E6%96%87%E4%BB%B6-prometheus-deploy-yaml-%E6%B3%A8%E6%84%8F%E7%89%88%E6%9C%AC%E9%9C%80%E8%A6%81%E7%94%A82-20"><span class="toc-number">1.2.2.3.</span> <span class="toc-text">prometheus部署文件  prometheus-deploy.yaml(注意版本需要用2.20)</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#prometheus%E9%85%8D%E7%BD%AE%E6%8A%A5%E8%AD%A6%E8%A7%84%E5%88%99-prometheus-rules-yaml"><span class="toc-number">1.2.2.4.</span> <span class="toc-text">prometheus配置报警规则 prometheus-rules.yaml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#node-exporter%E9%85%8D%E7%BD%AEnode-exporter-yaml-%E6%B3%A8%E6%84%8F%E7%89%88%E6%9C%AC%E9%9C%80%E8%A6%81%E7%94%A81-0-1"><span class="toc-number">1.2.2.5.</span> <span class="toc-text">node-exporter配置node-exporter.yaml(注意版本需要用1.0.1)</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#alertmanager-yaml%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.3.</span> <span class="toc-text">alertmanager yaml文件</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#alertmanager%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6alertmanger-configmap-yaml"><span class="toc-number">1.2.3.1.</span> <span class="toc-text">alertmanager配置文件alertmanger-configmap.yaml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#alertmanager-template%E6%96%87%E4%BB%B6alertmanager-template-yaml"><span class="toc-number">1.2.3.2.</span> <span class="toc-text">alertmanager template文件alertmanager-template.yaml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#alertmanager%E9%83%A8%E7%BD%B2%E6%96%87%E4%BB%B6alertmanager-deployment-yaml"><span class="toc-number">1.2.3.3.</span> <span class="toc-text">alertmanager部署文件alertmanager-deployment.yaml</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#grafana-yaml%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.4.</span> <span class="toc-text">grafana yaml文件</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%83%A8%E7%BD%B2%E5%88%B0k8s%E4%B8%AD"><span class="toc-number">1.2.5.</span> <span class="toc-text">部署到k8s中</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#grafana%E6%95%B0%E6%8D%AE%E6%BA%90%E5%92%8C%E7%9B%91%E6%8E%A7"><span class="toc-number">1.3.</span> <span class="toc-text">grafana数据源和监控</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#grafana%E6%B7%BB%E5%8A%A0%E6%95%B0%E6%8D%AE%E6%BA%90"><span class="toc-number">1.3.1.</span> <span class="toc-text">grafana添加数据源</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#import%E5%AF%BC%E5%85%A5%E6%A8%A1%E6%9D%BF"><span class="toc-number">1.3.2.</span> <span class="toc-text">import导入模板</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BF%AE%E6%94%B9prometheus-rules%E9%AA%8C%E8%AF%81%E7%9B%91%E6%8E%A7%E8%A7%A6%E5%8F%91%E6%8A%A5%E8%AD%A6%E5%B9%B6%E5%8F%91%E9%80%81%E9%82%AE%E4%BB%B6"><span class="toc-number">1.3.3.</span> <span class="toc-text">修改prometheus rules验证监控触发报警并发送邮件</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%84%9F%E8%B0%A2%E5%A4%A7%E4%BD%AC"><span class="toc-number">1.4.</span> <span class="toc-text">感谢大佬</span></a></li></ol></li></ol></div></div></div><div class="card-widget card-recent-post"><div class="card-content"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/posts/43435.html" title="创建lvm格式磁盘分区"><img src="https://img.alexcld.com/img/girl3.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="创建lvm格式磁盘分区"/></a><div class="content"><a class="title" href="/posts/43435.html" title="创建lvm格式磁盘分区">创建lvm格式磁盘分区</a><time datetime="2021-08-12T06:02:22.000Z" title="发表于 2021-08-12 14:02:22">2021-08-12</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/63478.html" title="k8s+jenkins实现CI，CD"><img src="https://img.alexcld.com/img/222823-160337690344c5.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="k8s+jenkins实现CI，CD"/></a><div class="content"><a class="title" href="/posts/63478.html" title="k8s+jenkins实现CI，CD">k8s+jenkins实现CI，CD</a><time datetime="2021-07-28T06:07:46.000Z" title="发表于 2021-07-28 14:07:46">2021-07-28</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/6543.html" title="alertmanager钉钉报警"><img src="https://img.alexcld.com/img/nvpu3.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="alertmanager钉钉报警"/></a><div class="content"><a class="title" href="/posts/6543.html" title="alertmanager钉钉报警">alertmanager钉钉报警</a><time datetime="2021-07-27T06:37:35.000Z" title="发表于 2021-07-27 14:37:35">2021-07-27</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/267647.html" title="监控体系部署之后走上了grafana模板踩坑之路"><img src="https://img.alexcld.com/img/195251-16080331718ccc.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="监控体系部署之后走上了grafana模板踩坑之路"/></a><div class="content"><a class="title" href="/posts/267647.html" title="监控体系部署之后走上了grafana模板踩坑之路">监控体系部署之后走上了grafana模板踩坑之路</a><time datetime="2021-07-22T05:33:31.000Z" title="发表于 2021-07-22 13:33:31">2021-07-22</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/24142.html" title="Prometheus邮件告警自定义模板的触发时间为utc的解决方法"><img src="https://img.alexcld.com/img/193615-1568028975fe7b.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Prometheus邮件告警自定义模板的触发时间为utc的解决方法"/></a><div class="content"><a class="title" href="/posts/24142.html" title="Prometheus邮件告警自定义模板的触发时间为utc的解决方法">Prometheus邮件告警自定义模板的触发时间为utc的解决方法</a><time datetime="2021-07-21T09:30:06.000Z" title="发表于 2021-07-21 17:30:06">2021-07-21</time></div></div></div></div></div></div></div></main><footer id="footer" style="background-image: url(https://img.alexcld.com/img/214606-15553359668cc5.jpg)"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By Alex</div><div class="framework-info"><span> </span><a target="_blank" rel="noopener" href="https://github.com/alexclownfish">Github</a><span class="footer-separator">|</span><span> </span><a target="_blank" rel="noopener" href="https://me.csdn.net/weixin_45509582">CSDN</a></div><div class="footer_custom_text"><span id="runtime"></span></div><div class="icp"><a target="_blank" rel="noopener" href="https://beian.miit.gov.cn/"><img class="icp-icon" src="/img/icp.png" alt="ICP"/><span>豫ICP备2020032561号-1</span></a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="chat_btn" type="button" title="rightside.chat_btn"><i class="fas fa-sms"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"><div id="local-hits"></div><div id="local-stats"><div class="local-search-stats__hr" id="hr"><span>由</span> <a target="_blank" rel="noopener" href="https://github.com/wzpan/hexo-generator-search" style="color:#49B1F5;">hexo-generator-search</a>
 <span>提供支持</span></div></div></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js"></script><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><div class="js-pjax"><script>function loadValine () {
  function initValine () {
    let initData = {
      el: '#vcomment',
      appId: 'rdtTOtMkfivtPvkQaaDXmJUd-gzGzoHsz',
      appKey: 'E68vw0GUqAynKmIiHtw9FLRA',
      placeholder: '留下你的看法吧！！',
      avatar: 'monsterid',
      meta: 'nick,mail,link'.split(','),
      pageSize: '10',
      lang: 'zh-CN',
      recordIP: true,
      serverURLs: '',
      emojiCDN: '',
      emojiMaps: "",
      enableQQ: true,
      path: window.location.pathname,
    }

    if (true) { 
      initData.requiredFields= ('nick,mail'.split(','))
    }
    
    if (false) {
      const otherData = false
      initData = Object.assign({}, initData, otherData)
    }
    
    const valine = new Valine(initData)
  }

  if (typeof Valine === 'function') initValine() 
  else $.getScript('https://cdn.jsdelivr.net/npm/valine/dist/Valine.min.js', initValine)
}

if ('Valine' === 'Valine' || !false) {
  if (false) btf.loadComment(document.querySelector('#vcomment'),loadValine)
  else setTimeout(() => loadValine(), 0)
} else {
  function loadOtherComment () {
    loadValine()
  }
}</script><script>function loadTwikoo () {
  function init () {
    twikoo.init({ 
      envId: '' 
    })
  }

  if (typeof twikoo.init === 'function') {
    init()
  } else {
    $.getScript('https://cdn.jsdelivr.net/npm/twikoo/dist/twikoo.all.min.js', init)
  }
}

if ('Valine' === 'Twikoo' || !false) {
  if (false) btf.loadComment(document.getElementById('twikoo'), loadTwikoo)
  else loadTwikoo()
} else {
  function loadOtherComment () {
    loadTwikoo()
  }
}</script><script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><script src="/js/flipcountdown.js"></script><script data-pjax src="/js/runtime.js"></script><script src="https://cdn.jsdelivr.net/gh/sviptzk/StaticFile_HEXO@latest/butterfly/js/pool.min.js"></script><script src="https://cdn.jsdelivr.net/npm/vue@2.6.11"></script><script src="https://cdn.jsdelivr.net/gh/alexclownfish/hexo-gitcalendar@v1.0/gitcalendar.js"></script><script src="https://apip.weatherdt.com/simple/static/js/weather-simple-common.js?v=2.0"></script><script src="/js/alexweather.js"></script><script src="/js/timing.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/activate-power-mode.min.js"></script><script>POWERMODE.colorful = true;
POWERMODE.shake = true;
POWERMODE.mobile = false;
document.body.addEventListener('input', POWERMODE);
</script><script id="click-show-text" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-show-text.min.js" async="async" mobile="false"></script></div><script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"model":{"jsonPath":"/live2dw/assets/tororo.model.json"},"display":{"position":"left","width":150,"height":300},"mobile":{"show":true},"log":false,"pluginJsPath":"lib/","pluginModelPath":"assets/","pluginRootPath":"live2dw/","tagMode":false});</script></body></html>